package com.lvmama.pangolin.timer.process;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.List;

import net.sf.json.JSONObject;

import org.apache.log4j.Logger;
import org.quartz.JobExecutionContext;
import org.springframework.beans.factory.annotation.Autowired;

import com.lvmama.pangolin.busi.config.ConfigManager;
import com.lvmama.pangolin.common.IPTools;
import com.lvmama.pangolin.common.JSONUtil;
import com.lvmama.pangolin.common.StringTools;
import com.lvmama.pangolin.entity.IP;
import com.lvmama.pangolin.proxyip.biz.DefaultCollectProxyIP;
import com.lvmama.pangolin.proxyip.inter.impl.GouBanJiaProcesser;
import com.lvmama.pangolin.proxyip.inter.impl.HttpsDaiLiProcesser;
import com.lvmama.pangolin.proxyip.inter.impl.IP181Processer;
import com.lvmama.pangolin.proxyip.inter.impl.IP3366Processer;
import com.lvmama.pangolin.proxyip.inter.impl.IP66Processer;
import com.lvmama.pangolin.proxyip.inter.impl.KuaiDaiLiProcesser;
import com.lvmama.pangolin.proxyip.inter.impl.MiMiIPProcesser;
import com.lvmama.pangolin.proxyip.inter.impl.NianShaoProxyIPProcesser;
import com.lvmama.pangolin.proxyip.inter.impl.XiCiDaiLiProcesser;
import com.lvmama.pangolin.service.IPService;
import com.lvmama.pangolin.timer.CommonJob;
import com.lvmama.pangolin.util.HttpsUtil;
import com.lvmama.pangolin.util.SpringContextUtil;


public class GrabProxyIP extends CommonJob
{

	Logger logger = Logger.getLogger(GrabProxyIP.class);
	@Autowired
	private IPService iPService;
	@Override
	public void runJob(JobExecutionContext context, String parameter)
	{
		
		logger.info("抓取网上代理IP数据开始！");
		if(iPService == null)
		{
			iPService = (IPService) SpringContextUtil.getBean(IPService.class);
		}
		try 
		{
			
            List<IP> ipList = null;
            DefaultCollectProxyIP collectProxyIP = null;
            
//	            http://www.xicidaili.com/ 调用有限制
            collectProxyIP = new DefaultCollectProxyIP("http://www.xicidaili.com/nn/", 1);
            ipList = collectProxyIP.doRequest(new XiCiDaiLiProcesser());
            storeProxyIP(ipList);

            // http://www.ip181.com/
            collectProxyIP = new DefaultCollectProxyIP("http://www.ip181.com/daili/", ".html", 1);
            ipList = collectProxyIP.doRequest(new IP181Processer());
            storeProxyIP(ipList);

            // http://www.nianshao.me/
            collectProxyIP = new DefaultCollectProxyIP("http://www.nianshao.me/?stype=1",false);
            ipList = collectProxyIP.doRequest(new NianShaoProxyIPProcesser());
            storeProxyIP(ipList);
            
            // http://www.nianshao.me/
            collectProxyIP = new DefaultCollectProxyIP("http://www.nianshao.me/?stype=2",false);
            ipList = collectProxyIP.doRequest(new NianShaoProxyIPProcesser());
            storeProxyIP(ipList);
            

            // http://www.66ip.cn/
            collectProxyIP = new DefaultCollectProxyIP("http://www.66ip.cn/areaindex_2/", ".html", 1, 1);
            ipList = collectProxyIP.doRequest(new IP66Processer());
            storeProxyIP(ipList);

            // http://www.httpsdaili.com/  error
            collectProxyIP = new DefaultCollectProxyIP("http://www.httpsdaili.com/?stype=1&page=", 1);
            ipList = collectProxyIP.doRequest(new HttpsDaiLiProcesser());
            storeProxyIP(ipList);

            // http://www.ip3366.net/ 速度过快
            collectProxyIP = new DefaultCollectProxyIP("http://www.ip3366.net/?page=", 1);
            ipList = collectProxyIP.doRequest(new IP3366Processer());
            storeProxyIP(ipList);

            //http://www.kuaidaili.com/
            collectProxyIP = new DefaultCollectProxyIP("http://www.kuaidaili.com/proxylist/", "/", 1);
            ipList = collectProxyIP.doRequest(new KuaiDaiLiProcesser());
            storeProxyIP(ipList);

            //http://www.mimiip.com/gngao/
            collectProxyIP = new DefaultCollectProxyIP("http://www.mimiip.com/gngao/", 1);
            ipList = collectProxyIP.doRequest(new MiMiIPProcesser());
            storeProxyIP(ipList);



            // http://www.goubanjia.com/
            collectProxyIP = new DefaultCollectProxyIP("http://www.goubanjia.com/free/gngn/index.shtml", false);
            ipList = collectProxyIP.doRequest(new GouBanJiaProcesser());
            storeProxyIP(ipList);

            collectProxyIP = new DefaultCollectProxyIP("http://www.goubanjia.com/free/gngn/index2.shtml", false);
            ipList = collectProxyIP.doRequest(new GouBanJiaProcesser());
            storeProxyIP(ipList);
            
            collectProxyIP = new DefaultCollectProxyIP("http://www.goubanjia.com/free/type/https/index.shtml", false);
            ipList = collectProxyIP.doRequest(new GouBanJiaProcesser());
            storeProxyIP(ipList);
            
            

            logger.info("成功插入代理IP数据！");
	        } catch (Exception e) {
	        	logger.error("抓取网上代理IP数据异常",e);
	        }
		logger.info("抓取网上代理IP数据结束！");
		
	}
	
	private String queryIpAttribution(String ip) throws UnsupportedEncodingException
	{ 
		String  ret = null;
		String url = "http://int.dpool.sina.com.cn/iplookup/iplookup.php?format=Json&ip=";
		String json = HttpsUtil.requestGet(url+ip);
		if(StringTools.isValid(json))
		{
			JSONObject jo = JSONUtil.getObject(json);
			ret = URLDecoder.decode(jo.getString("country"),"UTF-8");
		}
		return ret;
		
	}
	private void storeProxyIP(List<IP> ipList) 
	{
		if(ipList != null && !ipList.isEmpty())
		{
			boolean valid = false;
			for (IP ip : ipList)
			{
				try
				{
					if(ip.getAnonymity() != null && ip.getAnonymity().contains("高匿")
							&& "中国".equals(queryIpAttribution(ip.getIp())))
					{
						String filterKey = ConfigManager.getInstance().getParamValue("PROXY_IP_FILTER_CONFIG", "PROXY_IP_FILTER_CONFIG", "true");
						valid = IPTools.checkIPValid(ip.getIp(), ip.getIp_port(),1000,filterKey);						
						if(valid)
						{
							ip.setAvailable(1);
							iPService.insertProxyIP(ip);
							logger.info("插入成功!" );
						}
					}
					
					
				}catch(Exception e)
				{
					logger.error("验证IP有效性异常或IP入表失败！IP:"+ip.getIp() 
							+ " PORT:"+ip.getIp_port(),e);
				}
				
				
			}
			
		}
	}
	
}
